In [130]:
import time
for a_mdl in all_final_models:
xval_perf = a_mdl.model_performance(xval=True)
time.sleep(3)
valid_perf = a_mdl.model_performance(valid=True)
print('Model ID: ', a_mdl.model_id)
print('XVal Accuracy: ', xval_perf.accuracy()[0][1])
print('Training time (ms): ', a_mdl.run_time)
print('XVal AUC: ', xval_perf.auc())
time.sleep(3)
xval_perf.plot()
# print('Validation data AUC: ', valid_perf.auc())
# print('Validation data Accuracy: ', valid_perf.accuracy()[0][1])
time.sleep(3)
# valid_perf.plot()
print("-----------------------------")
Model ID: DeepLearning_model_python_1605423034668_341 XVal Accuracy: 0.8511856091578087 Training time (ms): 470361 XVal AUC: 0.9027482269503546
----------------------------- Model ID: DeepLearning_model_python_1605362597414_3340 XVal Accuracy: 0.7710547833197057 Training time (ms): 92889 XVal AUC: 0.8209884751773049
----------------------------- Model ID: DRF_model_python_1605423034668_386 XVal Accuracy: 0.856909239574816 Training time (ms): 1900902 XVal AUC: 0.9106057919621748
----------------------------- Model ID: DRF_model_python_1605362597414_2415 XVal Accuracy: 0.8348323793949305 Training time (ms): 4970 XVal AUC: 0.8844089834515367
----------------------------- Model ID: GBM_model_python_1605423034668_39 XVal Accuracy: 0.8479149632052331 Training time (ms): 71120 XVal AUC: 0.9054151891252956
----------------------------- Model ID: GBM_model_python_1605362597414_1525 XVal Accuracy: 0.8274734260016353 Training time (ms): 22756 XVal AUC: 0.8782254728132387
----------------------------- Model ID: GLM_model_python_1605423034668_20 XVal Accuracy: 0.8577269010629599 Training time (ms): 6048 XVal AUC: 0.9118498817966902
----------------------------- Model ID: GLM_model_python_1605362597414_1506 XVal Accuracy: 0.8086672117743254 Training time (ms): 216 XVal AUC: 0.8693484042553191
----------------------------- Model ID: NaiveBayes_model_python_1605423034668_1 XVal Accuracy: 0.7334423548650858 Training time (ms): 5211 XVal AUC: 0.7146276595744682
----------------------------- Model ID: NaiveBayes_model_python_1605362597414_1487 XVal Accuracy: 0.6745707277187244 Training time (ms): 130 XVal AUC: 0.62177304964539
----------------------------- Model ID: XGBoost_model_python_1605423034668_274 XVal Accuracy: 0.866721177432543 Training time (ms): 9638 XVal AUC: 0.9238918439716312
----------------------------- Model ID: XGBoost_model_python_1605362597414_1768 XVal Accuracy: 0.83892068683565 Training time (ms): 4348 XVal AUC: 0.8880614657210402
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning XVal Accuracy: 0.8086672117743254 Training time (ms): 54195 XVal AUC: 0.8633983451536642
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning XVal Accuracy: 0.7391659852820932 Training time (ms): 280253 XVal AUC: 0.7148921394799055
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf XVal Accuracy: 0.8683565004088307 Training time (ms): 16388 XVal AUC: 0.9260268912529551
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf XVal Accuracy: 0.8242027800490597 Training time (ms): 7132 XVal AUC: 0.880774231678487
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm XVal Accuracy: 0.866721177432543 Training time (ms): 14372 XVal AUC: 0.9249010047281323
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm XVal Accuracy: 0.8086672117743254 Training time (ms): 12611 XVal AUC: 0.8498921394799054
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm XVal Accuracy: 0.874897792313982 Training time (ms): 13136 XVal AUC: 0.9231072695035462
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm XVal Accuracy: 0.8405560098119379 Training time (ms): 685 XVal AUC: 0.9017479314420803
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes XVal Accuracy: 0.8634505314799673 Training time (ms): 12785 XVal AUC: 0.9184530141843972
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes XVal Accuracy: 0.8454619787408013 Training time (ms): 655 XVal AUC: 0.8984471040189125
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost XVal Accuracy: 0.8503679476696647 Training time (ms): 17681 XVal AUC: 0.9159352836879433
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost XVal Accuracy: 0.8250204415372036 Training time (ms): 5528 XVal AUC: 0.8839967494089835
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning XVal Accuracy: 0.7481602616516762 Training time (ms): 47602 XVal AUC: 0.8082505910165484
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning XVal Accuracy: 0.7269010629599346 Training time (ms): 279786 XVal AUC: 0.7530082742316785 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf XVal Accuracy: 0.8659035159443991 Training time (ms): 11379 XVal AUC: 0.9191282505910165 ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf XVal Accuracy: 0.821749795584628 Training time (ms): 6638 XVal AUC: 0.8804920212765958 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm XVal Accuracy: 0.866721177432543 Training time (ms): 11276 XVal AUC: 0.9223847517730496 ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm XVal Accuracy: 0.8062142273098937 Training time (ms): 11894 XVal AUC: 0.854057328605201 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm XVal Accuracy: 0.8691741618969746 Training time (ms): 9850 XVal AUC: 0.9235534869976358
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm XVal Accuracy: 0.8413736713000818 Training time (ms): 597 XVal AUC: 0.8965174349881797
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes XVal Accuracy: 0.8618152085036794 Training time (ms): 9295 XVal AUC: 0.9221498226950354
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes XVal Accuracy: 0.8405560098119379 Training time (ms): 601 XVal AUC: 0.8956368203309693
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost XVal Accuracy: 0.8528209321340965 Training time (ms): 14069 XVal AUC: 0.9068676122931443
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost XVal Accuracy: 0.8233851185609158 Training time (ms): 5439 XVal AUC: 0.8786244089834515
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning XVal Accuracy: 0.8282910874897792 Training time (ms): 45733 XVal AUC: 0.8788741134751773
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning XVal Accuracy: 0.7031888798037612 Training time (ms): 338640 XVal AUC: 0.6323758865248227
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf XVal Accuracy: 0.8708094848732625 Training time (ms): 10068 XVal AUC: 0.9200059101654847
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf XVal Accuracy: 0.8160261651676206 Training time (ms): 6518 XVal AUC: 0.8663593380614658
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm XVal Accuracy: 0.8618152085036794 Training time (ms): 10280 XVal AUC: 0.9218543144208038
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm XVal Accuracy: 0.7923139820114473 Training time (ms): 11571 XVal AUC: 0.8412086288416076
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm XVal Accuracy: 0.866721177432543 Training time (ms): 8128 XVal AUC: 0.9242213356973995
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm XVal Accuracy: 0.8413736713000818 Training time (ms): 589 XVal AUC: 0.8967715721040189
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes XVal Accuracy: 0.8683565004088307 Training time (ms): 7497 XVal AUC: 0.9242686170212766
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes XVal Accuracy: 0.83892068683565 Training time (ms): 468 XVal AUC: 0.8972680260047281
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost XVal Accuracy: 0.8479149632052331 Training time (ms): 11570 XVal AUC: 0.9019355791962175
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost XVal Accuracy: 0.8143908421913328 Training time (ms): 6247 XVal AUC: 0.8744400118203309
-----------------------------
